    /*
     * Dataset.java
     *
     * Created on den 23 september 2007, 18:24
     *
     * To change this template, choose Tools | Template Manager
     * and open the template in the editor.
     */

package grex;


import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Hashtable;
import java.util.StringTokenizer;

/**
 *
 * @author RIK
 */
public class Dataset {
    private int rows = 0,nrOfColumns,targetColumnIndex=-1,nrOfOutputs=1,markerColumn=-1;
    private String[][] rawData=null;
    private ArrayList <String> fileData=new  ArrayList();
    private ArrayList <String> arlMarkerColumn;
    private double numData[][],normData[][];
    private String [] columnNames=null;
    private String fileName=null;
    private javax.swing.table.DefaultTableModel tableModel=null;
    public static String  TRAIN="TRAIN",TEST="TEST",VAL="VAL";
    private String [] IOITEMS = {"Input","Output","None"};
    public final int CONTINUOUS_COLUMN = 1, CATEGORY_COLUMN=2, NONE_COLUMN=3,MARKER_COLUMN=4;
    private int[] columnTypes;
    private static final int CATEGORY_BORDER=20; //If more than 20 values the column is used as a continous column (if not set in the construnctor)
    private double trainPercent=0.66,valPercent=0;
    private double[][] maxMin,trainingInputs,testInputs,validationInputs,trainingOutputs,testOutputs,validationOutputs;
    private static int MAX = 0, MIN = 1;
    private Hashtable[] hstColumnValues;
    private Hashtable[] hstTranslators;
    private Hashtable[] hstReverseTranslators;
    private String delimiter="\t";
    
    /** Creates a new instance of Dataset */
    public Dataset(String fileName, String delimiter, int[] columnTypes) throws ArrayIndexOutOfBoundsException,FileNotFoundException,IOException{
        if(delimiter!=null)
            this.delimiter = delimiter;
        loadFile(fileName,delimiter);
        //initDataValues();
        this.columnTypes=columnTypes;
    }
    public Dataset(String fileName, String delimiter) throws ArrayIndexOutOfBoundsException,FileNotFoundException,IOException{
        if(delimiter!=null)
            this.delimiter = delimiter;
        loadFile(fileName,delimiter);
    }
    
    private void loadFile(String fileName, String delimiter) throws ArrayIndexOutOfBoundsException,FileNotFoundException,IOException{
        
        fileData = readFile(fileName,delimiter);
        columnNames = getColumnNames(fileData.get(0));
        if(columnNames != null)
            fileData.remove(0);
        else
            columnNames = createColumnNames(fileData.get(0).split(delimiter).length);
        rawData = translateFileDataToStringData(fileData,delimiter);     
        nrOfColumns = rawData[0].length;
        initColumnTypes();
        initStringToNumericalTranslators(rawData);
        numData = translateDataToNumericalFormat(rawData);
        maxMin = getMaxMin(numData);
        normData = normalizeData(numData);
    }
    public int getMarkerColumn(){
        return markerColumn;
    }

    public void setRawData(String[][] data,int[] columnTypes){
        rawData = data;
        nrOfColumns = rawData[0].length;
        this.columnTypes =columnTypes;
        initStringToNumericalTranslators(rawData);
        numData = translateDataToNumericalFormat(rawData);
        maxMin = getMaxMin(numData);
        normData = normalizeData(numData);
    }
    
    public double[][] getNormalizedTrainingInputs(){
        if(trainingInputs==null){            
            int[] columns = new int[nrOfColumns];            
            for (int i = 0; i < columns.length-1; i++) {
                if(columnTypes[i]!=NONE_COLUMN)
                columns[i]=i;
            }
            String[][] orgData = getOrgData(TRAIN,columns);
            trainingInputs=normalizeData(translateDataToNumericalFormat(orgData));
        }
        return trainingInputs;
    }

    public double[][] getNormalizedTestInputs(){
        if(testInputs==null){            
            int[] columns = new int[nrOfColumns];            
            for (int i = 0; i < columns.length-1; i++) {
                columns[i]=i;
            }
            String[][] orgData = getOrgData(TEST,columns);
            testInputs=normalizeData(translateDataToNumericalFormat(orgData));
        }
        return testInputs;
    }
    
    
    public double[][] getNormalizedValidationInputs(){
        if(validationInputs==null){            
            int[] columns = new int[nrOfColumns];            
            for (int i = 0; i < columns.length-1; i++) {
                if(columnTypes[i]!=NONE_COLUMN)
                    if(columnTypes[i]!=NONE_COLUMN)
                columns[i]=i;
            }
            String[][] orgData = getOrgData(VAL,columns);
            validationInputs=normalizeData(translateDataToNumericalFormat(orgData));
        }
        return validationInputs;
    }
    public double[][] getNormalizedTrainnOutputs(){
        if(trainingOutputs==null){            
            int[] columns = new int[nrOfOutputs];      
            for (int i = 0; i < nrOfOutputs; i++) {
                columns[i]=nrOfColumns+i;
            }
            String[][] orgData = getOrgData(TRAIN,columns);
            trainingOutputs=normalizeData(translateDataToNumericalFormat(orgData));
        }
        return trainingOutputs;
    }
     public double[][] getNormalizedValidationOutputs(){
        if(validationOutputs==null){            
            int[] columns = new int[nrOfColumns];            
            for (int i = 0; i < columns.length-1; i++) {
                columns[i]=i;
            }
            String[][] orgData = getOrgData(VAL,columns);
            validationInputs=normalizeData(translateDataToNumericalFormat(orgData));
        }
        return validationInputs;
    }
    public double[][] getNormalizedTestOutputs(){
        if(validationOutputs==null){            
            int[] columns = new int[nrOfColumns];            
            for (int i = 0; i < columns.length-1; i++) {
                columns[i]=i;
            }
            String[][] orgData = getOrgData(VAL,columns);
            validationInputs=normalizeData(translateDataToNumericalFormat(orgData));
        }
        return validationInputs;
    }
    
    private ArrayList <String> readFile(String fileName,String delimiter){
        ArrayList <String> tmpData = new ArrayList();
        try {
            BufferedReader b = new BufferedReader(new FileReader(fileName));
            String line="";
            
            while((line=b.readLine())!=null){                        
                if(line.charAt(0)!='%')
                    tmpData.add(line);
            }
        } catch (FileNotFoundException ex) {
            ex.printStackTrace();
        } catch (IOException ex) {
            ex.printStackTrace();
        }
        return tmpData;
    }
    
    /*Returns the part corresponding to the marker tag from the
     *orignial string data. int[] columns is an array of the column
     *indexes that should be returned. If column is null all columns
     *will be returned*/
    private String[][] getOrgData(String marker,int[] columns){
        ArrayList<String[]> tmpData = new ArrayList();
        for (int r = 0; r < normData.length; r++) {
            if(arlMarkerColumn.get(r).equalsIgnoreCase(marker)){
                if(columns==null || rawData[0].length==columns.length){
                    tmpData.add(rawData[r]);
                }else{
                    String[] rowData = new String[columns.length];
                    for (int c = 0; c < columns.length; c++) {
                        rowData[c]=rawData[r][c];
                    }
                    tmpData.add(rowData);
                }
            }
        }
        
        String[][] returnData = new String[tmpData.size()][columns.length];
        for(int r = 0; r<tmpData.size();r++){
            returnData[r]= tmpData.get(r);
        }
        
        return returnData;
    }
    
    private String[] getColumnNames(String nameRow){
        boolean hasColumnNames = true;
        StringTokenizer row = new StringTokenizer(nameRow);
        ArrayList <String> names = new ArrayList();
        for(int i=0;row.hasMoreElements();i++){
            Object token = row.nextElement();
            if(token instanceof Integer || token instanceof Double)
                return null;
        //    if(!token.toString().equalsIgnoreCase("TARGET"))
                names.add(token.toString());
        }
        return names.toArray(new String[names.size()]);
    }
    
    private String[] createColumnNames(int columns){
        String[] names = new String[columns];
        for (int i = 0; i < names.length-1; i++) {
            names[i]="X"+i;
        }
        names[names.length-1]="TARGET";
        return names;
    }
    
    
    /*Converts the file data to a strign array. If a markercolumn exists its is
     *removed and stored in arlMarkerColumn, if not arlMarkerColumns is created
     *with the porportions set of vadildationPrecent and TrainPercent;*/
    
    private String[][] translateFileDataToStringData(ArrayList<String> fileData,String delimiter) throws IOException{
        markerColumn=-1;
        StringTokenizer lastRow = new StringTokenizer(fileData.get(fileData.size()-1),delimiter);  
        int nrOfcolumns= lastRow.countTokens()+1; //+1 for att logaa till en column om marker column saknas.
        for(int k=0;lastRow.hasMoreTokens();k++){            
            String token = lastRow.nextToken();
            if(token.equalsIgnoreCase(TRAIN)||token.equalsIgnoreCase(VAL)||token.equalsIgnoreCase(TEST)){
                markerColumn=k;
                nrOfcolumns--; //tabort exra columnen om det redan finns en markercolumn
                break;
            }
        }
        
        rawData = new String [fileData.size()][nrOfcolumns];
        hstColumnValues = new Hashtable[nrOfcolumns];
        arlMarkerColumn = new ArrayList();
        for (int i = 0; i < hstColumnValues.length; i++) {
            hstColumnValues[i]=new Hashtable();
        }
        for (int r = 0; r < fileData.size(); r++) {
            StringTokenizer line = new StringTokenizer(fileData.get(r),delimiter);
            int vc = 0;//column for hstColumnValues (somtimes missing marker column
            for(int c = 0; line.hasMoreTokens();c++){
                String token = line.nextToken();
                if(c==markerColumn){
                    if(token.equalsIgnoreCase(TRAIN)||token.equalsIgnoreCase(VAL)||token.equalsIgnoreCase(TEST))
                        arlMarkerColumn.add(token);
                    else
                        throw new IOException(token + "(row"+r+"col"+c+"is not an allowed marker (TRAIN/VAL/TEST)" );
                }
                else{
                    hstColumnValues[vc].put(token,r);
                    rawData[r][vc]=token; //Using the vc counter removes marker column if present.
                    vc++;
                }
            }
        }
        if(arlMarkerColumn.size()==0){            
            createMarkerColumn();
            markerColumn=rawData[0].length-1;
            for(int i = 0; i< arlMarkerColumn.size();i++){ //stommer ej kolla so att columnerna or bortagna...                
                rawData[i][markerColumn] = arlMarkerColumn.get(i);
            }
        }
        return rawData;
    }
    
    private void initColumnTypes(){
        columnTypes= new int[hstColumnValues.length];
        Hashtable hstCurrent;
        for (int c = 0; c < hstColumnValues.length; c++) {
            System.out.println(c+"col: "+hstColumnValues[c].keySet().size());
            if(hstColumnValues[c].keySet().size()<=CATEGORY_BORDER)
                columnTypes[c]=CATEGORY_COLUMN;
            else
                columnTypes[c]=CONTINUOUS_COLUMN;
        }
        System.out.println("column types:");
        for (int i = 0; i < columnTypes.length; i++) {
            System.out.print(""+columnTypes[i]+" ");
        }
        System.out.println("");
    }
         /*Inits hstMarkerColumn by first adding the correct number of each
          *type and then shuffeling the order. Finally the generated values are
          *added to the hastable with the string as key and the index as value.*/
    private void createMarkerColumn(){
        arlMarkerColumn = new ArrayList();
        for (int i = 0; i < rawData.length; i++) {
            if(trainPercent*rawData.length>i)
                arlMarkerColumn.add(TRAIN);
            else if ((trainPercent+valPercent)*rawData.length>i)
                arlMarkerColumn.add(VAL);
            else
                arlMarkerColumn.add(TEST);
        }
        Collections.shuffle(arlMarkerColumn);
        columnNames = Arrays.copyOf(columnNames, columnNames.length+1);       
        columnNames[columnNames.length-1]="Marker";
                
    }
    //tabort marker column
    private double[][] translateDataToNumericalFormat(String[][] rawData){
        double [][] numData= new double[rawData.length][rawData[0].length];
        for (int c = 0; c < rawData[0].length; c++) {
            for (int r = 0; r < fileData.size(); r++) {
                if(columnTypes[c]==CATEGORY_COLUMN){
                    Double translation=(Double) hstTranslators[c].get(rawData[r][c]);
                    if(translation == null){
                        throw new NumberFormatException("New data contains category values that didn't exist in orginal data. Col:"+c+" Row" + r + " value:"+rawData[r][c]);
                    }
                    numData[r][c] = translation;
                }else{
                    try{
                        numData[r][c]=Double.parseDouble(rawData[r][c]);
                    }catch(Exception e){
                        throw new NumberFormatException("Column " + c +" is interpret sa an continuous column in original data but now contains the non numerical value: " + rawData[r][c]+".");
                    }
                }
                
            }
        }//*/
        return numData;
    }
    
    private void initStringToNumericalTranslators(String[][] rawData){        
        hstTranslators = new Hashtable[rawData[0].length];
        hstReverseTranslators = new Hashtable[rawData[0].length];
        for (int c = 0; c < rawData[0].length; c++) {
            if(columnTypes[c]==CATEGORY_COLUMN){
                hstTranslators[c]=new Hashtable();
                hstReverseTranslators[c]=new Hashtable();
            }
            for (int r = 0; r < fileData.size(); r++) {
                if(columnTypes[c]==CATEGORY_COLUMN){
                    Double translation=(Double) hstTranslators[c].get(rawData[r][c]);
                    if(translation == null){
                        translation = new Double(hstTranslators[c].keySet().size());
                        hstTranslators[c].put(rawData[r][c],translation);
                        hstReverseTranslators[c].put(translation,rawData[r][c]);
                    }
                }
            }
        }
    }
    
    private String[][] translateNumDataToOriginalFormat(double[][] numData){
        String[][] orgData = new String[numData.length][numData[0].length];
        
        //hstTranslators
        for (int r = 0; r < orgData.length; r++) {
            for (int c = 0; c < orgData[0].length; c++) {
                if(columnTypes[c]==CATEGORY_COLUMN){
                    String translation= "" +hstReverseTranslators[c].get(numData[r][c]);
                    orgData[r][c] = translation;
                }else{
                    orgData[r][c] = "" + numData[r][c];
                }//*/
            }
        }
        return orgData;
    }
    
    // Max min bor beroknas po train och val
    private double[][] getMaxMin(double[][] data){
        double[][] maxMin = new double[2][data[0].length];
        maxMin[MIN] = Arrays.copyOf(data[0],data[0].length);
        maxMin[MAX] = Arrays.copyOf(data[0],data[0].length);
        for (int r = 0; r < data.length; r++) {
            for (int c = 0; c < data[0].length; c++) {
                if(maxMin[MIN][c]>data[r][c])
                    maxMin[MIN][c]=data[r][c];
                if(maxMin[MAX][c]<data[r][c])
                    maxMin[MAX][c]=data[r][c];
            }
        }
        return maxMin;
    }
    
    private double[][] normalizeData(double [][] numData){
        double[][] normData = new double[numData.length][numData[0].length];
        for (int r = 0; r < normData.length; r++) {
            for (int c = 0; c < normData[0].length; c++) {
                normData[r][c]=(numData[r][c]-maxMin[MIN][c])/(maxMin[MAX][c]-maxMin[MIN][c]);
            }
        }
        return normData;
    }
    
    public double[][] unNormalizeData(double [][] normData){
        double[][] unNormData = new double[normData.length][normData[0].length];
        for (int r = 0; r < normData.length; r++) {
            for (int c = 0; c < normData[0].length; c++) {
                if(columnTypes[c]==CATEGORY_COLUMN)
                    unNormData[r][c]=Math.round(normData[r][c]*(maxMin[MAX][c]-maxMin[MIN][c])+maxMin[MIN][c]);
                else
                    unNormData[r][c]=(normData[r][c]*(maxMin[MAX][c]-maxMin[MIN][c])+maxMin[MIN][c]);
            }
        }
        return unNormData;
    }
    
    public String reTranslateNumericValue(double value,int column){
        if(hstReverseTranslators[column]!=null){
           return (String) hstReverseTranslators[column].get(new Double(value)); 
        } else {
            return ""+value;
        }        
    }
    
    public void printNumericData(){
        //System.out.println(rawData.length + " " + rawData[0].length);
        for (int i = 0; i < numData.length; i++) {
            for (int j = 0; j < numData[0].length; j++) {
                System.out.print(numData[i][j]+ " ");
            }
            System.out.println("");
        }//*/
    }
    
    public void printNormData(){
        //System.out.println(rawData.length + " " + rawData[0].length);
        for (int i = 0; i < normData.length; i++) {
            for (int j = 0; j < normData[0].length; j++) {
                System.out.print(normData[i][j]+ " ");
            }
            System.out.println("");
        }//*/
    }
    
    public void printUnNormalizedData(){
        double [][] org =  unNormalizeData(normData);
        for (int i = 0; i < org.length; i++) {
            for (int j = 0; j < org[0].length; j++) {
                System.out.print(org[i][j]+ " ");
            }
            System.out.println("");
        }//*/
    }
    
    public void printOrgData(){
        String [][] org =  translateNumDataToOriginalFormat(unNormalizeData(normData));
        for (int i = 0; i < org.length; i++) {
            for (int j = 0; j < org[0].length; j++) {
                System.out.print(org[i][j]+ " ");
            }
            System.out.println("");
        }//*/
    }
    
    public Object[][] getRawData(){
        return rawData;
    }
    public String[] getColumnNames(){
        return columnNames;
    }
    
}
